package com.jobcn.spider;

import com.jobcn.common.util.StringUtil;
import com.jobcn.config.SpiderConfig;
import com.jobcn.dic.SpiderDic;
import com.jobcn.service.SpiderService;
import com.jobcn.service.SpiderServiceImpl;
import com.jobcn.service.SpringUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.selector.Selectable;

import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

/**
 * 页面的分析器
 * Created by winson on 17-6-20.
 */
@Service
public class Processor58 implements PageProcessor {
    final Logger logger = LoggerFactory.getLogger(Processor58.class);

    @Autowired
    private SpiderConfig spiderConfig;
    @Autowired
    private SpiderService spiderService;
    private Integer taskId; //任务Id
    private List<String> seedUrlList; //存放拼装后的种子url
    private Site site;

    public Processor58() {
        super();
    }

    public Processor58(int taskId, List<String> seedUrlList) {
        this.taskId = taskId;
        this.seedUrlList = seedUrlList;
        if (spiderConfig == null) {
            spiderConfig = SpringUtil.getBean(SpiderConfig.class);
        }

        site = Site.me()
                .setUserAgent(spiderConfig.getUserAgent())
                .setCharset(spiderConfig.getCharset())
                .setRetryTimes(spiderConfig.getRetryTimes()) //就我的经验,这个重试一般用处不大
                .setTimeOut(spiderConfig.getTimeOut())//在使用代理的情况下,这个需要设置,可以考虑调大线程数目
                .setSleepTime(spiderConfig.getSleepTimes())//使用代理了之后,代理会通过切换IP来防止反扒。同时,使用代理本身qps降低了,所以这个可以小一些
                .setCycleRetryTimes(spiderConfig.getCycleRetryTime())//这个重试会换IP重试,是setRetryTimes的上一层的重试,不要怕三次重试解决一切问题。。
                .setUseGzip(true)
                .setDisableCookieManagement(true);
    }

    private Integer getTaskId() {
        return this.taskId;
    }

    @Override
    public Site getSite() {
        return site;
    }

    final static String REGEX_SERVICE_INFO = "http://\\w+.58.com/\\w+/\\d{14}x.shtml";
    final static String REGEX_SERVICE_INFO_URL = "(http://\\w+.58.com/\\w+/\\d{14}x.shtml).*";
    final static String REGEX_SERVICE_INFO_CHECK_URL = "http://(\\w+.58.com/\\w+)/\\d{14}x.shtml.*";
    final static String REGEX_SERVICE_INFO_ID = "http://\\w+.58.com/\\w+/(\\d{14})x.shtml";
    final static String REGEX_SERVICE_LIST = "http://\\w+.58.com/\\w+/pn\\d+/\\?renzheng=1";
    final static String REGEX_MER_SHOP = "http://shop.58.com/\\d+";
    final static String REGEX_MER_SHOP_SERVICE_LIST = "http://shop.58.com/\\d+/product";
    final static String REGEX_MER_ID = "http://shop.58.com/(\\d+).*";
    final static String REGEX_MER_MAP = "http://shop.58.com/mapbar/mapIframe\\?id=\\d+";
    final static String REGEX_MER_MAP_id = "http://shop.58.com/mapbar/mapIframe\\?id=(\\d+)";
    final static String REGEX_SEED_URL = "http://\\w+.58.com/\\w+/\\?renzheng=1";
    final static String REGEX_MER_CREDIT_TARGET = "(http://credit.vip.58.com/usercredit\\?userId=\\d+).*";
    final static String REGEX_MER_CREDIT = "http://credit.vip.58.com/usercredit\\?userId=\\d+";
    final static String REGEX_MER_CREDIT_ID = "http://shop.58.com/(\\d+)/photo/";

    @Override
    // process是定制爬虫逻辑的核心接口，在这里编写抽取逻辑
    public void process(Page page) {
        ////===============判断是否跳过该页面的处理：例如被拦截要求输验证码的页面，没有相关信息的页面===============
        //判断是否出验证码
        String error = page.getHtml().xpath("//div[@class='pop']/p[@class='title']/text()").toString();
        if (error != null && error.indexOf("验证码") > 0) {
            logger.error("请求被拦截:{}", page.getUrl());
            page.setSkip(true);
            if (spiderService == null)
                spiderService = SpringUtil.getBean(SpiderService.class);
            spiderService.spider58Stop();
        }
        //判断是否找不到相关信息
        //if ("ERROR".equals(page.getHtml().xpath("//div[@class='content clearfix']/div[@class='head']/p[@class='et']/text()"))) {
        //    logger.info("没有找到相关信息:{}", page.getUrl());
        //    page.setSkip(true);
        //}

        String url = page.getUrl().toString();
        ////===============判断页面类型，在分类型处理html页面===============
        if (url.matches(REGEX_SERVICE_INFO)) { //1：商家服务详情页面
            //这里map用来判断是否还有类型没有爬到，用于统计
            SpiderServiceImpl.checkMap.remove(page.getUrl().regex(REGEX_SERVICE_INFO_CHECK_URL).toString());
            //获取市级地区编号
            page.putField("infoId", page.getUrl().regex(REGEX_SERVICE_INFO_ID).toString());
            if (page.getResultItems().get("infoId") == null) {
                page.setSkip(true);
            } else {
                //目前只爬营业执照认证的类型
                if (page.getHtml().xpath("//span[@class='usertag-business']/text()").toString() == null) {
                    page.setSkip(true);
                }
                if (SpiderServiceImpl.addressCode.get(page.getUrl().regex("http://(\\w+).58.com/\\w+/\\d{14}x.shtml").toString())==null){
                    page.setSkip(true);
                }
                page.putField("addressCode", SpiderServiceImpl.addressCode.get(page.getUrl().regex("http://(\\w+).58.com/\\w+/\\d{14}x.shtml").toString()));//任务id
                page.putField("taskType", SpiderDic.SPIDER_TYPE_SER_INFO);//任务id
                page.putField("url", url);//服务名
                page.putField("taskId", getTaskId());
                page.putField("infoName", page.getHtml().xpath("//div[@class='mainTitle']/h1/text()").toString());
                page.putField("infoDescription", page.getHtml().xpath("//div[@class='descriptionBox']/tidyText()").toString());
                page.putField("infoUpdatetime", page.getHtml().xpath("//ul[@class='mtit_con_left fl']/li[@class='time']/text()").toString());
                page.putField("infoPrice", page.getHtml().xpath("//span[@class='pos_salary']/text()").toString());
                //详情描述的list
                List<Selectable> details = page.getHtml().xpath("//div[@class='col_sub sumary no_col_left']/ul[@class='suUl']/li").nodes();
                for (Selectable detail : details) {
                    String title = detail.xpath("//div[@class='su_tit']/text()").toString();
                    if (title != null) {
                        switch (title) {
                            case "商家地址：":
                                List<String> addressList = detail.xpath("//div[@class='su_con']/a/text()").all();
                                addressList.add(detail.xpath("//div[@class='su_con']/span/text()").toString());
                                if (addressList.size() > 0) {
                                    List<String> addressListNew = new ArrayList<String>();
                                    //处理多余的字符
                                    for (String address : addressList) {
                                        if (address != null) {
                                            if (address.indexOf("到手机") >= 0)
                                                continue;
                                            addressListNew.add(address.replace("-", "").trim());
                                        }
                                    }
                                    page.putField("address", StringUtil.list2string(addressListNew));
                                }
                                break;
                            case "联 系 人：":
                                page.putField("linkName", detail.xpath("//div[@class='su_con']/a[1]/text()").toString());
                                page.putField("merId", detail.xpath("//div[@class='su_con']/a[1]/@href").toString());
                                break;
                            case "服务区域：":
                                List<String> areaList = detail.xpath("//div[@class='su_con quyuline']/a/text()").all();
                                page.putField("serviceArea", StringUtil.list2string(areaList));
                                break;
                            case "类       别：":
                                page.putField("infoClass", detail.xpath("//div[@class='su_con']/text()").toString());
                                break;
                            default:
                                break;
                        }
                    }
                }
            }
        } else if (url.matches(REGEX_MER_SHOP)) { //3：商家的店铺首页
            //认证类型5
            String cert = page.getHtml().xpath("//div[@class='qyMiaoshu']/ul/li[@class=contrz]/span[@class='icoyy']/@title").toString();
            if (cert != null) {
                switch (cert) {
                    case "营业执照认证":  //目前只爬营业执照认证的类型
                        page.putField("certType", SpiderDic.CERT_TYPE_COMPANY);
                        page.addTargetRequest("mapbar/mapIframe?id=" + page.getUrl().regex(REGEX_MER_ID).toString());
                        break;
                    case "身份证认证":
                        page.putField("certType", SpiderDic.CERT_TYPE_PEASON);
                        page.setSkip(true);
                        break;
                    default:
                        page.putField("certType", SpiderDic.CERT_TYPE_NULL);
                        page.setSkip(true);
                        break;
                }
            } else {
                page.putField("certType", SpiderDic.CERT_TYPE_NULL);
                page.setSkip(true);
            }
            page.addTargetRequest("mapbar/mapIframe?id=" + page.getUrl().regex(REGEX_MER_ID).toString());
            page.putField("merId", page.getUrl().regex(REGEX_MER_ID).toString());
            page.putField("taskType", SpiderDic.SPIDER_TYPE_MER_INFO);
            page.putField("taskId", getTaskId());
            page.putField("url", url);
            page.putField("merName", page.getHtml().xpath("//div[@class='fendianItem']/h3/text()").toString());
            page.putField("address", page.getHtml().xpath("//div[@class='fendianItem']/p[1]/text()").toString());
            page.putField("phone", page.getHtml().xpath("//div[@class='fendianItem']/p[@class='telNoBox']/b/text()").toString());
            page.putField("merDescription", page.getHtml().xpath("//div[@id='all_summary_con']/table/tbody/tr/td/text()").toString());
        } else if (url.matches(REGEX_MER_MAP)) {
            page.putField("merId", page.getUrl().regex(REGEX_MER_MAP_id).toString());
            page.putField("taskType", SpiderDic.SPIDER_TYPE_MER_MAP);
            page.putField("taskId", getTaskId());
            List<Selectable> scriptList = page.getHtml().xpath("//script").nodes();
            String script = scriptList.get(3).toString();
            String longitude = script.substring(script.indexOf("longitude"), script.lastIndexOf("var latitude")).trim();
            String latitude = script.substring(script.indexOf("latitude"), script.lastIndexOf("//var map")).trim();
            longitude = StringUtil.sub(longitude, "longitude = ", ";");
            latitude = StringUtil.sub(latitude, "latitude = ", ";");
            page.putField("longitude", longitude);
            page.putField("latitude", latitude);
        } else if (url.matches(REGEX_MER_CREDIT)) {
            if (page.getHtml().xpath("//i[@class='idCard_yes']") != null) {
                String merShopUrl = page.getHtml().xpath("//div[@class='album']/div[@class='bar']/a/@href").regex(REGEX_MER_CREDIT_ID).toString();
                if (merShopUrl != null) {
                    page.putField("merId", merShopUrl);
                    page.putField("taskType", SpiderDic.SPIDER_TYPE_MER_CREDIT);
                    page.putField("taskId", getTaskId());
                    page.putField("companyName", page.getHtml().xpath("//span[@class='headTitle']/text()").toString());
                    String registerDateTime = page.getHtml().xpath("//div[@class='headRight']/span/text()").regex("注册时间：(.*)").toString();
                    try {
                        page.putField("registerDatetime", new SimpleDateFormat("yyyy-MM-dd").parse(registerDateTime));
                    } catch (ParseException e) {
                        e.printStackTrace();
                    }
                    //注册号
                    List<Selectable> conList = page.getHtml().xpath("//div[@class='nav']/ul/li").nodes();
                    if (conList.size() > 1) {
                        List<Selectable> creditList = page.getHtml().xpath("//td").nodes();
                        for (int i = 0; i < creditList.size(); i++) {
                            if ("注册号".equals(creditList.get(i).xpath("//td/text()").toString())) {
                                page.putField("creditCode", creditList.get(i + 1).xpath("//td/text()").toString());
                                break;
                            }
                        }
                    }
                }
            }
        } else {
            page.setSkip(true);
        }

        //===============从页面发现后续的url地址来抓取===============
        //商家首页
        if (url.matches(REGEX_SERVICE_INFO)) {  //服务信息
            if (page.getHtml().xpath("//span[@class='usertag-business']/text()").toString() != null) {
                List<Selectable> userinfoList = page.getHtml().xpath("//div[@class='userinfo-link']/a").nodes();
                for (Selectable selectable : userinfoList) {
                    String type = selectable.xpath("//a/text()").toString();
                    switch (type) {
                        case "查看信用档案":
                            page.addTargetRequest(selectable.xpath("//a/@href").regex(REGEX_MER_CREDIT_TARGET).toString());
                            break;
                        case "进入店铺":
                            page.addTargetRequest(selectable.xpath("//a/@href").regex(REGEX_MER_SHOP).toString());
                            page.addTargetRequest(selectable.xpath("//a/@href").regex(REGEX_MER_SHOP).toString() + "/product");
                            break;
                        default:
                            break;
                    }
                }
            }
        } else if (url.matches(REGEX_SERVICE_LIST) || url.matches(REGEX_SEED_URL)) {   //服务列表
            if (url.matches(REGEX_SERVICE_LIST)) {
                logger.info("Spider:{}_{}_{}",page.getUrl().regex("http://\\w+.58.com/\\w+/pn(\\d+)/\\?renzheng=1"),
                        page.getUrl().regex("http://(\\w+).58.com/\\w+/pn\\d+/\\?renzheng=1"),
                        page.getUrl().regex("http://\\w+.58.com/(\\w+)/pn\\d+/\\?renzheng=1")
                        );
            } else {
                logger.info("Spider:{}_{}_{}",1,
                        page.getUrl().regex("http://(\\w+).58.com/\\w+/\\?renzheng=1"),
                        page.getUrl().regex("http://\\w+.58.com/(\\w+)/\\?renzheng=1"));
            }
            //logger.info("Spider:{}",page.getUrl());
            String nextPage = page.getHtml().xpath("//div[@class='pager']/a[@class='next']/@href").toString();
            if (nextPage != null) {
                nextPage = url.substring(0, url.indexOf("com")) + "com" + nextPage;
                page.addTargetRequest(nextPage);
            }
            List<String> list = page.getHtml().links().regex(REGEX_SERVICE_INFO_URL).all();
            for (String urlTemp:list){
                Matcher matcher = Pattern.compile("http://(\\w+).58.com/(\\w+)/\\d{14}x.shtml").matcher(urlTemp);
                while (matcher.find()) {
                    if (SpiderServiceImpl.addressCode.get(matcher.group(1))!=null&&SpiderServiceImpl.sortCode.get(matcher.group(2))!=null){
                        page.addTargetRequest(urlTemp);
                    }
                }
            }
            //page.addTargetRequests(page.getHtml().links().regex(REGEX_SERVICE_INFO_URL).all());
        } else if (url.matches(REGEX_MER_SHOP_SERVICE_LIST)) { //商家商铺的列表
            List<String> list = page.getHtml().links().regex(REGEX_SERVICE_INFO_URL).all();
            for (String urlTemp:list){
                Matcher matcher = Pattern.compile("http://(\\w+).58.com/(\\w+)/\\d{14}x.shtml").matcher(urlTemp);
                while (matcher.find()) {
                    if (SpiderServiceImpl.addressCode.get(matcher.group(1))!=null&&SpiderServiceImpl.sortCode.get(matcher.group(2))!=null){
                        page.addTargetRequest(urlTemp);
                    }
                }
            }
            //page.addTargetRequests(page.getHtml().links().regex(REGEX_SERVICE_INFO_URL).all());
        }

    }

}